Merge remote-tracking branch 'origin/master' into js_agent_can_be_dryrun

Andrew Cantino преди 9 години
родител
ревизия
ea1fb447ca

+ 7 - 1
.env.example

@@ -147,7 +147,7 @@ ENABLE_SECOND_PRECISION_SCHEDULE=false
147 147
 # Specify the scheduler frequency in seconds (default: 0.3).
148 148
 # Increasing this value will help reduce the use of system resources
149 149
 # at the expense of time accuracy.
150
-#SCHEDULER_FREQUENCY=0.3
150
+SCHEDULER_FREQUENCY=0.3
151 151
 
152 152
 # Use Graphviz for generating diagrams instead of using Google Chart
153 153
 # Tools.  Specify a dot(1) command path built with SVG support
@@ -159,3 +159,9 @@ TIMEZONE="Pacific Time (US & Canada)"
159 159
 
160 160
 # Number of failed jobs to keep in the database
161 161
 FAILED_JOBS_TO_KEEP=100
162
+
163
+# Maximum runtime of background jobs in minutes
164
+DELAYED_JOB_MAX_RUNTIME=20
165
+
166
+# Amount of seconds for delayed_job to sleep before checking for new jobs
167
+DELAYED_JOB_SLEEP_DELAY=10

+ 32 - 25
Gemfile

@@ -58,6 +58,7 @@ gem 'daemons', '~> 1.1.9'
58 58
 gem 'delayed_job', '~> 4.0.0'
59 59
 gem 'delayed_job_active_record', '~> 4.0.0'
60 60
 gem 'devise', '~> 3.4.0'
61
+gem 'dotenv-rails', '~> 2.0.1'
61 62
 gem 'em-http-request', '~> 1.1.2'
62 63
 gem 'faraday', '~> 0.9.0'
63 64
 gem 'faraday_middleware'
@@ -96,27 +97,25 @@ group :development do
96 97
   gem 'guard'
97 98
   gem 'guard-livereload'
98 99
   gem 'guard-rspec'
99
-end
100 100
 
101
-group :development, :test do
102
-  gem 'coveralls', require: false
103
-  gem 'delorean'
104
-  gem 'dotenv-rails'
105
-  gem 'pry'
106
-  gem 'rr'
107
-  gem 'rspec', '~> 3.2'
108
-  gem 'rspec-collection_matchers', '~> 1.1.0'
109
-  gem 'rspec-rails', '~> 3.1'
110
-  gem 'rspec-html-matchers', '~> 0.7'
111
-  gem 'shoulda-matchers'
112
-  gem 'spring', '~> 1.3.0'
113
-  gem 'spring-commands-rspec'
114
-  gem 'vcr'
115
-  gem 'webmock', '~> 1.17.4', require: false
101
+  group :test do
102
+    gem 'coveralls', require: false
103
+    gem 'delorean'
104
+    gem 'pry'
105
+    gem 'rr'
106
+    gem 'rspec', '~> 3.2'
107
+    gem 'rspec-collection_matchers', '~> 1.1.0'
108
+    gem 'rspec-rails', '~> 3.1'
109
+    gem 'rspec-html-matchers', '~> 0.7'
110
+    gem 'shoulda-matchers'
111
+    gem 'spring', '~> 1.3.0'
112
+    gem 'spring-commands-rspec'
113
+    gem 'vcr'
114
+    gem 'webmock', '~> 1.17.4', require: false
115
+  end
116 116
 end
117 117
 
118 118
 group :production do
119
-  gem 'dotenv-deployment'
120 119
   gem 'rack'
121 120
 end
122 121
 
@@ -126,15 +125,23 @@ gem 'tzinfo', '>= 1.2.0'	# required by rails; 1.2.0 has support for *BSD and Sol
126 125
 # Windows does not have zoneinfo files, so bundle the tzinfo-data gem.
127 126
 gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw]
128 127
 
129
-# This hack needs some explanation.  When on Heroku, use the pg, unicorn, and rails12factor gems.
130
-# When not on Heroku, we still want our Gemfile.lock to include these gems, so we scope them to
131
-# an unsupported platform.
132
-if ENV['ON_HEROKU'] || ENV['HEROKU_POSTGRESQL_ROSE_URL'] || ENV['HEROKU_POSTGRESQL_GOLD_URL'] || File.read(File.join(File.dirname(__FILE__), 'Procfile')) =~ /intended for Heroku/
128
+# Introduces a scope for Heroku specific gems.
129
+def on_heroku
130
+  if ENV['ON_HEROKU'] ||
131
+     ENV['HEROKU_POSTGRESQL_ROSE_URL'] ||
132
+     ENV['HEROKU_POSTGRESQL_GOLD_URL'] ||
133
+     File.read(File.join(File.dirname(__FILE__), 'Procfile')) =~ /intended for Heroku/
134
+    yield
135
+  else
136
+    # When not on Heroku, we still want our Gemfile.lock to include
137
+    # Heroku specific gems, so we scope them to an unsupported
138
+    # platform.
139
+    platform :ruby_18, &proc
140
+  end
141
+end
142
+
143
+on_heroku do
133 144
   gem 'pg'
134 145
   gem 'unicorn'
135 146
   gem 'rails_12factor', group: :production
136
-else
137
-  gem 'pg', platform: :ruby_18
138
-  gem 'unicorn', platform: :ruby_18
139
-  gem 'rails_12factor', platform: :ruby_18
140 147
 end

+ 4 - 7
Gemfile.lock

@@ -124,11 +124,9 @@ GEM
124 124
     docile (1.1.5)
125 125
     domain_name (0.5.24)
126 126
       unf (>= 0.0.5, < 1.0.0)
127
-    dotenv (0.11.1)
128
-      dotenv-deployment (~> 0.0.2)
129
-    dotenv-deployment (0.0.2)
130
-    dotenv-rails (0.11.1)
131
-      dotenv (= 0.11.1)
127
+    dotenv (2.0.1)
128
+    dotenv-rails (2.0.1)
129
+      dotenv (= 2.0.1)
132 130
     dropbox-api (0.4.2)
133 131
       hashie
134 132
       multi_json
@@ -507,8 +505,7 @@ DEPENDENCIES
507 505
   delayed_job_active_record (~> 4.0.0)
508 506
   delorean
509 507
   devise (~> 3.4.0)
510
-  dotenv-deployment
511
-  dotenv-rails
508
+  dotenv-rails (~> 2.0.1)
512 509
   dropbox-api
513 510
   em-http-request (~> 1.1.2)
514 511
   faraday (~> 0.9.0)

+ 26 - 0
app/assets/javascripts/components/utils.js.coffee

@@ -33,3 +33,29 @@ class @Utils
33 33
       onHide?()
34 34
     body?(modal.querySelector('.modal-body'))
35 35
     $(modal).modal('show')
36
+
37
+  @handleDryRunButton: (button, data = $(button.form).serialize()) ->
38
+    $(button).prop('disabled', true)
39
+    $('body').css(cursor: 'progress')
40
+    $.ajax type: 'POST', url: $(button).data('action-url'), dataType: 'json', data: data
41
+      .always =>
42
+        $('body').css(cursor: 'auto')
43
+      .done (json) =>
44
+        Utils.showDynamicModal """
45
+          <h5>Log</h5>
46
+          <pre class="agent-dry-run-log"></pre>
47
+          <h5>Events</h5>
48
+          <pre class="agent-dry-run-events"></pre>
49
+          <h5>Memory</h5>
50
+          <pre class="agent-dry-run-memory"></pre>
51
+          """,
52
+          body: (body) ->
53
+            $(body).
54
+              find('.agent-dry-run-log').text(json.log).end().
55
+              find('.agent-dry-run-events').text(json.events).end().
56
+              find('.agent-dry-run-memory').text(json.memory)
57
+          title: 'Dry Run Results',
58
+          onHide: -> $(button).prop('disabled', false)
59
+      .fail (xhr, status, error) ->
60
+        alert('Error: ' + error)
61
+        $(button).prop('disabled', false)

+ 1 - 26
app/assets/javascripts/pages/agent-edit-page.js.coffee

@@ -174,32 +174,7 @@ class @AgentEditPage
174 174
 
175 175
   invokeDryRun: (e) =>
176 176
     e.preventDefault()
177
-    button = e.target
178
-    $(button).prop('disabled', true)
179
-    $('body').css(cursor: 'progress')
180
-    @updateFromEditors()
181
-    $.ajax type: 'POST', url: $(button).data('action-url'), dataType: 'json', data: $(button.form).serialize()
182
-      .always =>
183
-        $("body").css(cursor: 'auto')
184
-      .done (json) =>
185
-        Utils.showDynamicModal """
186
-          <h5>Log</h5>
187
-          <pre class="agent-dry-run-log"></pre>
188
-          <h5>Events</h5>
189
-          <pre class="agent-dry-run-events"></pre>
190
-          <h5>Memory</h5>
191
-          <pre class="agent-dry-run-memory"></pre>
192
-          """,
193
-          body: (body) ->
194
-            $(body).
195
-              find('.agent-dry-run-log').text(json.log).end().
196
-              find('.agent-dry-run-events').text(json.events).end().
197
-              find('.agent-dry-run-memory').text(json.memory)
198
-          title: 'Dry Run Results',
199
-          onHide: -> $(button).prop('disabled', false)
200
-      .fail (xhr, status, error) ->
201
-        alert('Error: ' + error)
202
-        $(button).prop('disabled', false)
177
+    Utils.handleDryRunButton(this)
203 178
 
204 179
 $ ->
205 180
   Utils.registerPage(AgentEditPage, forPathsMatching: /^agents/)

+ 0 - 4
app/assets/stylesheets/diagram.css.scss

@@ -8,13 +8,9 @@
8 8
   }
9 9
 
10 10
   .overlay-container {
11
-    position: absolute;
12
-    top: 0;
13
-    left: 0;
14 11
     z-index: auto;
15 12
 
16 13
     .overlay {
17
-      position: relative;
18 14
       z-index: auto;
19 15
       width: 100%;
20 16
       height: 100%;

+ 4 - 0
app/concerns/dry_runnable.rb

@@ -25,6 +25,10 @@ module DryRunnable
25 25
     )
26 26
   end
27 27
 
28
+  def dry_run?
29
+    is_a? Sandbox
30
+  end
31
+
28 32
   module Sandbox
29 33
     attr_accessor :results
30 34
 

+ 6 - 3
app/controllers/agents_controller.rb

@@ -35,15 +35,18 @@ class AgentsController < ApplicationController
35 35
   end
36 36
 
37 37
   def dry_run
38
-    attrs = params[:agent]
38
+    attrs = params[:agent] || {}
39 39
     if agent = current_user.agents.find_by(id: params[:id])
40 40
       # PUT /agents/:id/dry_run
41
-      type = agent.type
41
+      if attrs.present?
42
+        type = agent.type
43
+        agent = Agent.build_for_type(type, current_user, attrs)
44
+      end
42 45
     else
43 46
       # POST /agents/dry_run
44 47
       type = attrs.delete(:type)
48
+      agent = Agent.build_for_type(type, current_user, attrs)
45 49
     end
46
-    agent = Agent.build_for_type(type, current_user, attrs)
47 50
     agent.name ||= '(Untitled)'
48 51
 
49 52
     if agent.valid?

+ 9 - 5
app/controllers/diagrams_controller.rb

@@ -1,9 +1,13 @@
1 1
 class DiagramsController < ApplicationController
2 2
   def show
3
-    @agents = if params[:scenario_id].present?
4
-                current_user.scenarios.find(params[:scenario_id]).agents.includes(:receivers)
5
-              else
6
-                current_user.agents.includes(:receivers)
7
-              end
3
+    if params[:scenario_id].present?
4
+      @scenario = current_user.scenarios.find(params[:scenario_id])
5
+      agents = @scenario.agents
6
+    else
7
+      agents = current_user.agents
8
+    end
9
+    @disabled_agents = agents.inactive
10
+    agents = agents.active if params[:exclude_disabled].present?
11
+    @agents = agents.includes(:receivers)
8 12
   end
9 13
 end

+ 2 - 1
app/models/agent.rb

@@ -60,7 +60,8 @@ class Agent < ActiveRecord::Base
60 60
   has_many :scenario_memberships, :dependent => :destroy, :inverse_of => :agent
61 61
   has_many :scenarios, :through => :scenario_memberships, :inverse_of => :agents
62 62
 
63
-  scope :active, -> { where(disabled: false) }
63
+  scope :active,   -> { where(disabled: false) }
64
+  scope :inactive, -> { where(disabled: true) }
64 65
 
65 66
   scope :of_type, lambda { |type|
66 67
     type = case type

+ 15 - 11
app/models/agents/imap_folder_agent.rb

@@ -6,13 +6,15 @@ module Agents
6 6
   class ImapFolderAgent < Agent
7 7
     cannot_receive_events!
8 8
 
9
+    can_dry_run!
10
+
9 11
     default_schedule "every_30m"
10 12
 
11 13
     description <<-MD
12 14
 
13 15
       The ImapFolderAgent checks an IMAP server in specified folders
14 16
       and creates Events based on new mails found since the last run.
15
-      In the first visit to a foler, this agent only checks for the
17
+      In the first visit to a folder, this agent only checks for the
16 18
       initial status and does not create events.
17 19
 
18 20
       Specify an IMAP server to connect with `host`, and set `ssl` to
@@ -45,8 +47,8 @@ module Agents
45 47
           specified, will be chosen as the "body" value in a created
46 48
           event.
47 49
 
48
-          Named captues will appear in the "matches" hash in a created
49
-          event.
50
+          Named captures will appear in the "matches" hash in a
51
+          created event.
50 52
 
51 53
       - "from", "to", "cc"
52 54
 
@@ -311,7 +313,7 @@ module Agents
311 313
 
312 314
         if boolify(interpolated['mark_as_read'])
313 315
           log 'Marking as read'
314
-          mail.mark_as_read
316
+          mail.mark_as_read unless dry_run?
315 317
         end
316 318
       }
317 319
     end
@@ -322,7 +324,7 @@ module Agents
322 324
       port = (Integer(port) if port.present?)
323 325
 
324 326
       log "Connecting to #{host}#{':%d' % port if port}#{' via SSL' if ssl}"
325
-      Client.open(host, port, ssl) { |imap|
327
+      Client.open(host, port: port, ssl: ssl) { |imap|
326 328
         log "Logging in as #{username}"
327 329
         imap.login(username, interpolated[:password])
328 330
 
@@ -437,8 +439,8 @@ module Agents
437 439
 
438 440
     class Client < ::Net::IMAP
439 441
       class << self
440
-        def open(host, port, ssl)
441
-          imap = new(host, port, ssl)
442
+        def open(host, *args)
443
+          imap = new(host, *args)
442 444
           yield imap
443 445
         ensure
444 446
           imap.disconnect unless imap.nil?
@@ -525,17 +527,19 @@ module Agents
525 527
 
526 528
       def has_attachment?
527 529
         @has_attachment ||=
528
-          begin
529
-            data = @client.uid_fetch(@uid, 'BODYSTRUCTURE').first
530
+          if data = @client.uid_fetch(@uid, 'BODYSTRUCTURE').first
530 531
             struct_has_attachment?(data.attr['BODYSTRUCTURE'])
532
+          else
533
+            false
531 534
           end
532 535
       end
533 536
 
534 537
       def fetch
535 538
         @parsed ||=
536
-          begin
537
-            data = @client.uid_fetch(@uid, 'BODY.PEEK[]').first
539
+          if data = @client.uid_fetch(@uid, 'BODY.PEEK[]').first
538 540
             Mail.read_from_string(data.attr['BODY[]'])
541
+          else
542
+            Mail.read_from_string('')
539 543
           end
540 544
       end
541 545
 

+ 6 - 0
app/views/agents/_action_menu.html.erb

@@ -5,6 +5,12 @@
5 5
     </li>
6 6
   <% end %>
7 7
 
8
+  <% if agent.can_dry_run? %>
9
+    <li>
10
+      <%= link_to icon_tag('glyphicon-refresh') + ' Dry Run', '#', 'data-action-url' => dry_run_agent_path(agent), tabindex: "-1", onclick: "Utils.handleDryRunButton(this, '_method=PUT')" %>
11
+    </li>
12
+  <% end %>
13
+
8 14
   <li>
9 15
     <%= link_to icon_tag('glyphicon-eye-open') + ' Show'.html_safe, agent_path(agent) %>
10 16
   </li>

+ 8 - 1
app/views/diagrams/show.html.erb

@@ -9,7 +9,14 @@
9 9
         <h2>Agent Event Flow</h2>
10 10
       </div>
11 11
       <div class="btn-group">
12
-        <%= link_to icon_tag('glyphicon-chevron-left') + ' Back'.html_safe, (params[:scenario_id] ? scenario_path(params[:scenario_id]) : agents_path), class: "btn btn-default" %>
12
+        <%= link_to icon_tag('glyphicon-chevron-left') + ' Back'.html_safe, (@scenario ? scenario_path(@scenario) : agents_path), class: "btn btn-default" %>
13
+        <% if (num_disabled = @disabled_agents.count).nonzero? -%>
14
+          <% if params[:exclude_disabled] %>
15
+            <%= link_to @scenario ? scenario_diagram_path(@scenario) : diagram_path, class: 'btn btn-default' do %><%= icon_tag('glyphicon-eye-open') %> Show <%= pluralize(num_disabled, 'disabled Agent') %><% end %>
16
+          <% else %>
17
+            <%= link_to @scenario ? scenario_diagram_path(@scenario, exclude_disabled: true) : diagram_path(exclude_disabled: true), class: 'btn btn-default' do %><%= icon_tag('glyphicon-eye-close') %> Hide <%= pluralize(num_disabled, 'disabled Agent') %><% end %>
18
+          <% end %>
19
+        <% end %>
13 20
       </div>
14 21
 
15 22
       <div class='digraph'>

+ 2 - 0
bin/threaded.rb

@@ -2,6 +2,8 @@ require 'thread'
2 2
 require 'huginn_scheduler'
3 3
 require 'twitter_stream'
4 4
 
5
+Rails.configuration.cache_classes = true
6
+
5 7
 STDOUT.sync = true
6 8
 STDERR.sync = true
7 9
 

+ 2 - 2
config/application.rb

@@ -4,10 +4,10 @@ require 'rails/all'
4 4
 
5 5
 Bundler.require(:default, Rails.env)
6 6
 
7
-Dotenv.overload File.expand_path('../../spec/env.test', __FILE__) if Rails.env.test?
8
-
9 7
 module Huginn
10 8
   class Application < Rails::Application
9
+    Dotenv.overload File.expand_path('../../spec/env.test', __FILE__) if Rails.env.test?
10
+
11 11
     # Settings in config/environments/* take precedence over those specified here.
12 12
     # Application configuration should go into files in config/initializers
13 13
     # -- all .rb files in that directory are automatically loaded.

+ 2 - 1
config/initializers/delayed_job.rb

@@ -1,9 +1,10 @@
1 1
 Delayed::Worker.destroy_failed_jobs = false
2 2
 Delayed::Worker.max_attempts = 5
3
-Delayed::Worker.max_run_time = 20.minutes
3
+Delayed::Worker.max_run_time = (ENV['DELAYED_JOB_MAX_RUNTIME'].presence || 20).to_i.minutes
4 4
 Delayed::Worker.read_ahead = 5
5 5
 Delayed::Worker.default_priority = 10
6 6
 Delayed::Worker.delay_jobs = !Rails.env.test?
7
+Delayed::Worker.sleep_delay = (ENV['DELAYED_JOB_SLEEP_DELAY'].presence || 10).to_f
7 8
 
8 9
 # Delayed::Worker.logger = Logger.new(Rails.root.join('log', 'delayed_job.log'))
9 10
 # Delayed::Worker.logger.level = Logger::DEBUG

+ 32 - 8
spec/concerns/dry_runnable_spec.rb

@@ -8,10 +8,10 @@ describe DryRunnable do
8 8
 
9 9
     def check
10 10
       log "Logging"
11
-      create_event payload: { test: "foo" }
11
+      create_event payload: { 'test' => 'foo' }
12 12
       error "Recording error"
13
-      create_event payload: { test: "bar" }
14
-      self.memory = { last_status: "ok" }
13
+      create_event payload: { 'test' => 'bar' }
14
+      self.memory = { 'last_status' => 'ok', 'dry_run' => dry_run? }
15 15
       save!
16 16
     end
17 17
   end
@@ -24,18 +24,41 @@ describe DryRunnable do
24 24
     }
25 25
   end
26 26
 
27
-  it "traps logging, event emission and memory updating" do
27
+  def counts
28
+    [users(:bob).agents.count, users(:bob).events.count, users(:bob).logs.count]
29
+  end
30
+
31
+  it "does not affect normal run, with dry_run? returning false" do
32
+    before = counts
33
+    after  = before.zip([0, 2, 2]).map { |x, d| x + d }
34
+
35
+    expect {
36
+      @agent.check
37
+      @agent.reload
38
+    }.to change { counts }.from(before).to(after)
39
+
40
+    expect(@agent.memory).to eq({ 'last_status' => 'ok', 'dry_run' => false })
41
+
42
+    payloads = @agent.events.reorder(:id).last(2).map(&:payload)
43
+    expect(payloads).to eq([{ 'test' => 'foo' }, { 'test' => 'bar' }])
44
+
45
+    messages = @agent.logs.reorder(:id).last(2).map(&:message)
46
+    expect(messages).to eq(['Logging', 'Recording error'])
47
+  end
48
+
49
+  it "traps logging, event emission and memory updating, with dry_run? returning true" do
28 50
     results = nil
29 51
 
30 52
     expect {
31 53
       results = @agent.dry_run!
54
+      @agent.reload
32 55
     }.not_to change {
33
-      [users(:bob).agents.count, users(:bob).events.count, users(:bob).logs.count]
56
+      [@agent.memory, counts]
34 57
     }
35 58
 
36 59
     expect(results[:log]).to match(/\AI, .+ INFO -- : Logging\nE, .+ ERROR -- : Recording error\n/)
37
-    expect(results[:events]).to eq([{ test: 'foo' }, { test: 'bar' }])
38
-    expect(results[:memory]).to eq({ "last_status" => "ok" })
60
+    expect(results[:events]).to eq([{ 'test' => 'foo' }, { 'test' => 'bar' }])
61
+    expect(results[:memory]).to eq({ 'last_status' => 'ok', 'dry_run' => true })
39 62
   end
40 63
 
41 64
   it "does not perform dry-run if Agent does not support dry-run" do
@@ -45,8 +68,9 @@ describe DryRunnable do
45 68
 
46 69
     expect {
47 70
       results = @agent.dry_run!
71
+      @agent.reload
48 72
     }.not_to change {
49
-      [users(:bob).agents.count, users(:bob).events.count, users(:bob).logs.count]
73
+      [@agent.memory, counts]
50 74
     }
51 75
 
52 76
     expect(results[:log]).to match(/\AE, .+ ERROR -- : Exception during dry-run. SandboxedAgent does not support dry-run: /)